{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prompt Optimization\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Sys prompt directory sys_prompts does not exist.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[1], line 4\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mrandom\u001b[39;00m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtextwrap\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m dedent\n\u001b[0;32m----> 4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtinytextgrad\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mprompt\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Prompt\n\u001b[1;32m      5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtinytextgrad\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mttg\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m optimize_prompt\n\u001b[1;32m      8\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mapply_prompt\u001b[39m(\n\u001b[1;32m      9\u001b[0m     prompt: Prompt,\n\u001b[1;32m     10\u001b[0m     inputs: \u001b[38;5;28mlist\u001b[39m[\u001b[38;5;28mstr\u001b[39m],\n\u001b[1;32m     11\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mlist\u001b[39m[\u001b[38;5;28mstr\u001b[39m]:\n",
      "File \u001b[0;32m~/src/witt3rd/ecaa/notebooks/.venv/lib/python3.11/site-packages/tinytextgrad/__init__.py:3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mllm\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m call_llm\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mprompt\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Prompt, load_prompt\n\u001b[0;32m----> 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mttg\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m TGD, Engine, OptimizationResult, TextLoss, Variable, optimize_prompt\n\u001b[1;32m      5\u001b[0m __all__ \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m      6\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcall_llm\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m      7\u001b[0m     \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mVariable\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     17\u001b[0m ]\n",
      "File \u001b[0;32m~/src/witt3rd/ecaa/notebooks/.venv/lib/python3.11/site-packages/tinytextgrad/ttg.py:17\u001b[0m\n\u001b[1;32m     15\u001b[0m SYS_PROMPT_DIR \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mgetenv(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSYS_PROMPT_DIR\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msys_prompts\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     16\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mexists(SYS_PROMPT_DIR):\n\u001b[0;32m---> 17\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSys prompt directory \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mSYS_PROMPT_DIR\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m does not exist.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     19\u001b[0m APPLY_GRADIENT_PROMPT \u001b[38;5;241m=\u001b[39m load_prompt(\n\u001b[1;32m     20\u001b[0m     name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapply_gradient\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     21\u001b[0m     prompt_dir\u001b[38;5;241m=\u001b[39mSYS_PROMPT_DIR,\n\u001b[1;32m     22\u001b[0m )\n\u001b[1;32m     24\u001b[0m CLEANUP_TEXT_PROMPT \u001b[38;5;241m=\u001b[39m load_prompt(\n\u001b[1;32m     25\u001b[0m     name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcleanup_text\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     26\u001b[0m     prompt_dir\u001b[38;5;241m=\u001b[39mSYS_PROMPT_DIR,\n\u001b[1;32m     27\u001b[0m )\n",
      "\u001b[0;31mValueError\u001b[0m: Sys prompt directory sys_prompts does not exist."
     ]
    }
   ],
   "source": [
    "import random\n",
    "from textwrap import dedent\n",
    "\n",
    "from tinytextgrad.prompt import Prompt\n",
    "from tinytextgrad.ttg import optimize_prompt\n",
    "\n",
    "\n",
    "def apply_prompt(\n",
    "  prompt: Prompt,\n",
    "  inputs: list[str],\n",
    ") -> list[str]:\n",
    "  \"\"\"\n",
    "  Calls the LLM with the prompt and inputs.\n",
    "  Returns a list of strings representing the generated text.\n",
    "  \"\"\"\n",
    "  results = [\n",
    "    prompt.call_llm(input) for input in inputs\n",
    "  ]\n",
    "  for i, result in enumerate(results):\n",
    "    print(f\"Input {i+1}:\")\n",
    "    print(inputs[i])\n",
    "    print(f\"Output {i+1}:\")\n",
    "    print(result)\n",
    "  return results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "initial_prompt = dedent(\"\"\"\n",
    "Given some text and a question, determine if the text\n",
    "contains the answer to the question\n",
    "\"\"\").strip()\n",
    "\n",
    "data = [\n",
    "  \"Text: The cat is on the mat. Question: Where is the cat?\",\n",
    "  \"Text: The sky is blue. Question: What color is the grass?\",\n",
    "  \"Text: Paris is the capital of France. Question: What is the capital of Germany?\",\n",
    "  \"Text: The Earth orbits the Sun. Question: What does the Earth orbit?\",\n",
    "  \"Text: Water freezes at 0 degrees Celsius. Question: At what temperature does water boil?\",\n",
    "  \"Text: Beethoven composed many symphonies. Question: Who composed the Fifth Symphony?\",\n",
    "  \"Text: Elephants are the largest land animals. Question: What is the largest land animal?\",\n",
    "  \"Text: Shakespeare wrote 'Romeo and Juliet.' Question: Who wrote 'Hamlet'?\",\n",
    "  \"Text: Humans have 206 bones. Question: How many bones do humans have?\",\n",
    "  \"Text: Coffee is typically grown in tropical regions. Question: Where is coffee typically grown?\",\n",
    "]\n",
    "inputs = random.sample(data, 5)\n",
    "\n",
    "result = optimize_prompt(\n",
    "  initial_prompt=initial_prompt,\n",
    "  prompt_inputs=inputs,\n",
    "  model=\"gpt-4o-mini\",\n",
    "  eval_model=\"gpt-4o\",\n",
    "  num_iterations=1,\n",
    ")\n",
    "\n",
    "print(\n",
    "  \"\\n\\nFinal optimized TEXT_CONTAINS_ANSWER_PROMPT:\"\n",
    ")\n",
    "print(result)\n",
    "\n",
    "TEXT_CONTAINS_ANSWER_PROMPT = result.to_prompt()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
